In [84]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from wordcloud import WordCloud, STOPWORDS
First import the dataset and preview the data, the data is about the words that every philosophers said. I want to find out which words they said more often.¶
In [85]:
df=pd.read_csv("C:/Users/Zero/Documents/GitHub/Fall2021-Project1-yule-yueli/data/philosophy_data.csv")
In [86]:
df.head(n=10)
Out[86]:
Here is the version of jupyter notebook¶
In [87]:
import sys
print(sys.version)
print(sys.executable)
Creat a new variable called n_txt（number of tokenized_txt) and find out the number of valid words in every sentence.¶
In [88]:
df['n_txt'] = list(map(len,map(eval,df.tokenized_txt)))
Then preview the new dataset¶
In [89]:
df.head(n=10)
Out[89]:
First see the distributions of 3 categories:title, author, school.¶
In [90]:
plt.figure(figsize=(16,6))
df['title'].value_counts().plot(kind='bar')
plt.title('title')
plt.grid()
plt.show()
In [91]:
plt.figure(figsize=(16,6))
df['author'].value_counts().plot(kind='bar')
plt.title('author')
plt.grid()
plt.show()
In [92]:
plt.figure(figsize=(16,6))
df['school'].value_counts().plot(kind='bar')
plt.title('school')
plt.grid()
plt.show()
Histgram of n_txt, the distribution of n_txt is right skewed¶
In [93]:
print(df.n_txt.describe())
plt.figure(figsize=(12,5))
df.n_txt.plot(kind='hist', bins=200)
plt.title('Tokens')
plt.grid()
plt.show()
violin plot about n_txt by schools: approximately same means¶
In [94]:
plt.figure(figsize=(16,5))
sns.violinplot(x='school', y='n_txt', data=df)
plt.title('Number of Tokens - School')
plt.grid()
plt.show()
WordClouds by school¶
In [95]:
stopwords = set(STOPWORDS)
In [96]:
schools = df.school.unique().tolist()
for sc in schools:
df_temp = df[df.school==sc]
print('School = ', sc.upper(), ':')
# render wordcloud
text = " ".join(txt for txt in df_temp.sentence_lowered)
wordcloud = WordCloud(stopwords=stopwords, max_font_size=60, max_words=600,
width = 600, height = 300,
background_color="white").generate(text)
plt.figure(figsize=(12,8))
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()
